Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Calculate average answers when all questions are numeric

Andrew Cantino 12 years ago
parent
1e8dcc65f0
commit
e284782363
2 changed files with 84 additions and 7 deletions
Show Diff Stats
  1. 28 3
      app/models/agents/human_task_agent.rb
  2. 56 4
      spec/models/agents/human_task_agent_spec.rb

+ 28 - 3
app/models/agents/human_task_agent.rb
View File 

@@ -166,6 +166,8 @@ module Agents
166 166
167 167 
         log "Looking at HIT #{hit_id}.  I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
168 168 
         if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
169 
+          payload = { :answers => assignments.map(&:answers) }
170 
+
169 171 
           if options[:take_majority] == "true"
170 172 
             counts = {}
171 173 
             options[:hit][:questions].each do |question|
 
@@ -177,14 +179,29 @@ module Agents
177 179 
               end
178 180 
               counts[question[:key]] = question_counts
179 181 
             end
182 
+            payload[:counts] = counts
183 
+
180 184 
             majority_answer = counts.inject({}) do |memo, (key, question_counts)|
181 185 
               memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
182 186 
               memo
183 187 
             end
184 
-            event = create_event :payload => { :answers => assignments.map(&:answers), :counts => counts, :majority_answer => majority_answer }
185 
-          else
186 
-            event = create_event :payload => { :answers => assignments.map(&:answers) }
188 
+            payload[:majority_answer] = majority_answer
189 
+
190 
+            if all_questions_are_numeric?
191 
+              average_answer = counts.inject({}) do |memo, (key, question_counts)|
192 
+                sum = divisor = 0
193 
+                question_counts.to_a.each do |num, count|
194 
+                  sum += num.to_s.to_f * count
195 
+                  divisor += count
196 
+                end
197 
+                memo[key] = sum / divisor.to_f
198 
+                memo
199 
+              end
200 
+              payload[:average_answer] = average_answer
201 
+            end
187 202 
           end
203 
+
204 
+          event = create_event :payload => payload
188 205 
           log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym])
189 206
190 207 
           assignments.each(&:approve!)
 
@@ -194,6 +211,14 @@ module Agents
194 211 
       end
195 212 
     end
196 213
214 
+    def all_questions_are_numeric?
215 
+      options[:hit][:questions].all? do |question|
216 
+        question[:selections].all? do |selection|
217 
+          selection[:key] == selection[:key].to_f.to_s || selection[:key] == selection[:key].to_i.to_s
218 
+        end
219 
+      end
220 
+    end
221 
+
197 222 
     def create_hit(event = nil)
198 223 
       payload = event ? event.payload : {}
199 224 
       title = Utils.interpolate_jsonpaths(options[:hit][:title], payload).strip

+ 56 - 4
spec/models/agents/human_task_agent_spec.rb
View File 

@@ -236,6 +236,11 @@ describe Agents::HumanTaskAgent do
236 236 
     describe "taking majority votes" do
237 237 
       before do
238 238 
         @checker.options[:take_majority] = "true"
239 
+        @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
240 
+        mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
241 
+      end
242 
+
243 
+      it "should take the majority votes of all questions" do
239 244 
         @checker.options[:hit][:questions][1] = {
240 245 
           :type => "selection",
241 246 
           :key => "age_range",
 
@@ -248,11 +253,7 @@ describe Agents::HumanTaskAgent do
248 253 
               { :key => ">50", :text => "Over 50 years old" }
249 254 
             ]
250 255 
         }
251 
-      end
252 256
253 
-      it "should take the majority votes of all questions" do
254 
-        @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
255 
-        mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
256 257 
         assignments = [
257 258 
           FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad", "age_range"=>"<50"}),
258 259 
           FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "age_range"=>">50"}),
 
@@ -277,6 +278,57 @@ describe Agents::HumanTaskAgent do
277 278
278 279 
         @checker.events.last.payload[:counts].should == { :sentiment => { :happy => 2, :sad => 1, :neutral => 1 }, :age_range => { :">50" => 3, :"<50" => 1 } }
279 280 
         @checker.events.last.payload[:majority_answer].should == { :sentiment => "happy", :age_range => ">50" }
281 
+        @checker.events.last.payload.should_not have_key(:average_answer)
282 
+
283 
+        @checker.memory[:hits].should == {}
284 
+      end
285 
+
286 
+      it "should also provide an average answer when all questions are numeric" do
287 
+        @checker.options[:hit][:questions] = [
288 
+          {
289 
+            :type => "selection",
290 
+            :key => "rating",
291 
+            :name => "Rating",
292 
+            :required => "true",
293 
+            :question => "Please select a rating:",
294 
+            :selections =>
295 
+              [
296 
+                { :key => "1", :text => "One" },
297 
+                { :key => "2", :text => "Two" },
298 
+                { :key => "3", :text => "Three" },
299 
+                { :key => "4", :text => "Four" },
300 
+                { :key => "5.1", :text => "Five Point One" }
301 
+              ]
302 
+          }
303 
+        ]
304 
+
305 
+        assignments = [
306 
+          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"1" }),
307 
+          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"3" }),
308 
+          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"5.1" }),
309 
+          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"2" }),
310 
+          FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"2" })
311 
+        ]
312 
+        hit = FakeHit.new(:max_assignments => 5, :assignments => assignments)
313 
+        mock(RTurk::Hit).new("JH3132836336DHG") { hit }
314 
+
315 
+        lambda {
316 
+          @checker.send :review_hits
317 
+        }.should change { Event.count }.by(1)
318 
+
319 
+        assignments.all? {|a| a.approved == true }.should be_true
320 
+
321 
+        @checker.events.last.payload[:answers].should == [
322 
+          { :rating => "1" },
323 
+          { :rating => "3" },
324 
+          { :rating => "5.1" },
325 
+          { :rating => "2" },
326 
+          { :rating => "2" }
327 
+        ]
328 
+
329 
+        @checker.events.last.payload[:counts].should == { :rating => { :"1" => 1, :"2" => 2, :"3" => 1, :"4" => 0, :"5.1" => 1 } }
330 
+        @checker.events.last.payload[:majority_answer].should == { :rating => "2" }
331 
+        @checker.events.last.payload[:average_answer].should == { :rating => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
280 332
281 333 
         @checker.memory[:hits].should == {}
282 334 
       end